package edu.hawaii.webspider;

import java.net.URL;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.logging.Logger;
import java.util.logging.Handler;
import java.util.logging.Formatter;
import java.util.logging.Level;
import java.util.logging.LogRecord;
import com.meterware.httpunit.GetMethodWebRequest;
import com.meterware.httpunit.WebConversation;
import com.meterware.httpunit.WebRequest;
import com.meterware.httpunit.WebResponse;
import com.meterware.httpunit.WebLink;

/**
 * Retrieves web pages from the Internet and collect statistics.
 * 
 * @author Philip Johnson
 * @author Randy Cox
 */
public class WebSpider {

  /**
   * Token used to indicate parameter option in command line arguments.
   */
  public static final String PARAM_TOKEN = "-";

  /**
   * Help command line option.
   */
  public static final String PARAM_HELP = PARAM_TOKEN + "help";

  /**
   * Test parse command line option.
   */
  public static final String PARAM_TEST_PARSE = PARAM_TOKEN + "testparse";

  /**
   * The total links command line option.
   */
  public static final String PARAM_TOTAL_LINKS = PARAM_TOKEN + "totallinks";

  /**
   * The most popular command line option.
   */
  public static final String PARAM_MOST_POPULAR = PARAM_TOKEN + "mostpopular";

  /**
   * Logging command line option.
   */
  public static final String PARAM_LOGGING = PARAM_TOKEN + "logging";

  /**
   * Default amount of pages to search before completion.
   */
  public static final Integer MAX_PAGE_DEFAULT = 100;

  /**
   * Maximum pages to search allowed.
   */
  public static final Integer MAX_PAGE_LIMIT = 1000;

  /**
   * The message printed with -help command line is entered.
   */
  public static final String HELP_STR = "Syntax:  $java -jar webspider-cox url [maxpages] " + "["
    + PARAM_HELP + "] " + "[" + PARAM_TEST_PARSE + "] " + "[" + PARAM_TOTAL_LINKS + "] " + "["
    + PARAM_MOST_POPULAR + "] " + "[" + PARAM_LOGGING + "]";

  /**
   * The resulting printed line with formating used with returning total links information.
   */
  public static final String TOTAL_LINKS_STR 
    = "The total number of links discoverd while crawling the first %d pages "
    + "accessable from %s is : %d";

  /**
   * The resulting printed line with formating used with return the most popular site information.
   */
  public static final String MOST_POPULAR_STR 
    = "The most popular page discovered is: %s with %d pages that point to it.";

  /**
   * The error string to be printed when URL connection error occurs.
   */
  public static final String ERROR_URL_STR = "Error accessing %s.";

  /**
   * List of string tags used to populate the result string when -testparse is entered. Each element
   * in array is concatenated to the result string followed by the value it describes. The result
   * string is populated in by the order in found in this array.
   */
  static final String[] RESULT_TAG_LIST = { "Parser", "isHelp", "isTestParse",
    "isTotalLinksOn", "isMostPopularOn", "isLoggingOn", "isMaxPagesOk", "isTargetUrlOk",
    "targetUrl", "maxPages", "isOk" };

  /**
   * Static place holder to hold an instance of the webspider class. Used to allow the static main
   * function to instantiate and store the webspider class.
   */
  static WebSpider webSpider = null;

  /**
   * Parses the command string for arguments.
   */
  public Parser parser;

  /**
   * Logs events when -logging is used.
   */
  private static Logger logger = null;

  /**
   * A WebSpider crawls the web and returns info.
   * The method to avoid httpunit throwing exceptions when encountering javascript is
   * attributed to Kevin English.
   * @author Randy Cox
   * @author Kevin English
   * @param args Arguments passed in on command line.
   */
  public WebSpider(String[] args) {
    parser = new Parser(args);
    com.meterware.httpunit.HttpUnitOptions.setExceptionsThrownOnScriptError(false);
    com.meterware.httpunit.HttpUnitOptions.setExceptionsThrownOnErrorStatus(false);
    com.meterware.httpunit.HttpUnitOptions.setScriptingEnabled(false);
  }

  /**
   * Based on parameters, this function will count links and/or find most popular page or Url.
   * 
   * @param args URL, max pages to search, and optional switches (see HELP_STR).
   * @throws Exception when bad URL is encountered, which shouldn't happen.
  * 
   */
  public static void main(String[] args) throws Exception {
    WebSpider.webSpider = new WebSpider(args);
    System.out.println(WebSpider.webSpider.run());
  }

  /**
   * Based on parsed parameters, counts links, finds most popular url, and/or logs events.
   * 
   * @return Resulting data string that can be printed to standard output.
   * @throws Exception when bad URL is encountered, which shouldn't happen.
   */
  public String run() throws Exception {

    String result = "";

    // Provide status of all flags and parsed data
    if (this.parser.isTestParseOn) {
      result = this.parser.toString();
      return result;
    }

    // Provide a help string
    if (this.parser.isHelpOn) {
      result = WebSpider.HELP_STR;
      return result;
    }

    // Sets logging flag and configures the logging utility
    if ((this.parser.isLoggingOn) && (logger == null)) {
      logger = Logger.getLogger("edu.hawaii.webspider");
      logger.setUseParentHandlers(false);
      Handler handler = new StandardHandler();
      logger.addHandler(handler);
    }

    // Connects to URL and recursively gets links and collects statistics
    UrlStats urlStats = this.getUrlStats();
    if (urlStats == null) {
      this.parser.isTargetUrlOk = false;
      this.parser.isOk = false;
    }

    // If error occurred, provide error string
    if (!this.parser.isOk) {
      Object parameters[] = { this.parser.targetUrl };
      result = String.format(ERROR_URL_STR, parameters);
      return result;
    }

    // If count links option is set, provide the count links display result string
    if (this.parser.isTotalLinksOn) {
      Object parameters[] = { this.parser.maxPages, this.parser.targetUrl, urlStats.linkCount };
      result = String.format(TOTAL_LINKS_STR, parameters);
    }

    // If most popular option is set, provide the most popular URL display result string
    if (this.parser.isMostPopularOn) {
      Object parameters[] = { urlStats.mostPopularUrl, urlStats.mostPopularLinkCount };
      if (result.length() > 0) {
        result = result + "\r\n";
      }
      result = result.concat(String.format(MOST_POPULAR_STR, parameters));
    }

    return result;
  }

  /**
   * Connects and retrieves URL pages recursively and updates statistics.
   * 
   * @return UrlStats Statistics information with total links and most popular url.
   * @throws Exception when bad URL is encountered, which shouldn't happen.
   */
  public UrlStats getUrlStats() throws Exception {

    UrlStats urlStats = new UrlStats();
    UrlList urlList = new UrlList();

    // Read in main URL and store links
    printLog("Retrieving " + this.parser.targetUrl);
    List<String> pagesToCheck = getPages();
    if (pagesToCheck == null) {
      printLog("Unable to retrieve " + this.parser.targetUrl);
      return null;
    }
    urlStats.linkCount = pagesToCheck.size();
    printLog("Found " + urlStats.linkCount.toString() + " links.");

    // Keep track of visited pages
    List<String> pagesVisited = new ArrayList<String>();
    pagesVisited.add(this.parser.targetUrl);

    // Read through list of stored links, adding links as we go, while collecting stats.
    while ((!pagesToCheck.isEmpty()) && (pagesVisited.size() <= this.parser.maxPages)) {
      String page = pagesToCheck.remove(0);
      urlList.add(page);

      if (pagesVisited.contains(page)) {
        continue;
      }

      WebSpider childSpider = new WebSpider(new String[] { page });
      printLog("Retrieving " + page);
      List<String> childPages = childSpider.getPages();
      if (childPages == null) {
        printLog("Unable to retrieve " + page);
        continue;
      }
      printLog("Found " + ((Integer) childPages.size()).toString() + " links.");
      urlStats.linkCount += childPages.size();
      pagesVisited.add(page);
      pagesToCheck.addAll(childPages);
    }

    // Keep track of popularity of URLs from left over queue
    for (String page : pagesToCheck) {
      urlList.add(page);
    }
    pagesToCheck.clear();

    // Populate statistics variable and return
    UrlItem urlItem = urlList.getMostPopular();
    urlStats.mostPopularUrl = urlItem.url;
    urlStats.mostPopularLinkCount = urlItem.count;
    return urlStats;
  }

  /**
   * Gets all URL links found the designated URL page.
   * 
   * @return List of URL links
  * @throws Exception when bad URL gets through to getURL(), which shouldn't happen.
*/
  public List<String> getPages() throws Exception {
    WebConversation wc = new WebConversation();
    List<String> pages = new ArrayList<String>();
    WebLink[] webLinks;
    try {
      WebRequest request = new GetMethodWebRequest(this.parser.targetUrl);
      WebResponse response = wc.getResponse(request);
      webLinks = response.getLinks();
    }
    catch (Exception e) {
      return null;
    }

    for (WebLink webLink : webLinks) {
      WebRequest wbTemp = webLink.getRequest();
      URL urlTemp;
      urlTemp = wbTemp.getURL();
      String urlString = urlTemp.toString();
      pages.add(urlString);
    }
    return pages;
  }

  /**
   * Send message to be logged.
   * 
   * @param message string to be logged.
   */
  public void printLog(String message) {
    if (this.parser.isLoggingOn) {
      logger.log(Level.INFO, message);
    }
  }

  /**
   * Parses the argument list from the command line and stores results.
   * 
   * @author rcox
   */
  public static class Parser {

    /**
     * Help flag.
     */
    public Boolean isHelpOn = false;
    
    /**
     * Test parse flag.
     */
    public Boolean isTestParseOn = false;
    
    /**
     * Total links flag.
     */
    public Boolean isTotalLinksOn = false;
    
    /**
     * Most popular flag.
     */
    public Boolean isMostPopularOn = false;
    
    /**
     * Logging flag.
     */
    public Boolean isLoggingOn = false;
    
    /**
     * Max pages is valid flag.
     */
    public Boolean isMaxPagesOk = true;
    
    /**
     * Target URL is valid flag.
     */
    public Boolean isTargetUrlOk = false;
    
    /**
     * The URL string.
     */
    public String targetUrl = "";
    
    /**
     * Maximum pages to traverse.
     */
    public Integer maxPages = MAX_PAGE_DEFAULT;
    
    /**
     * Health flag.
     */
    public Boolean isOk = false;

    /**
     * Parses the command line string and sets the flags and fields.
     * @param args Command line paramter string
     */
    public Parser(String[] args) {
      isOk = true;
      for (String s : args) {

        // Check if this argument is a parameter and which parameter
        if (s.substring(0, 1).equals(PARAM_TOKEN)) {
          if (s.equals(PARAM_HELP)) {
            this.isHelpOn = true;
            continue;
          }
          if (s.equals(PARAM_TEST_PARSE)) {
            this.isTestParseOn = true;
            continue;
          }
          if (s.equals(PARAM_TOTAL_LINKS)) {
            this.isTotalLinksOn = true;
            continue;
          }
          else if (s.equals(PARAM_MOST_POPULAR)) {
            this.isMostPopularOn = true;
            continue;
          }
          else if (s.equals(PARAM_LOGGING)) {
            this.isLoggingOn = true;
            continue;
          }
        }

        // Not parameter or URL, so should be a number
        int newMaxPages = 0;
        try {
          newMaxPages = Integer.parseInt(s);
          this.maxPages = newMaxPages;
          this.isMaxPagesOk = true;
          continue;
        }

        // Not a number, so should be a URL
        catch (NumberFormatException e) {
          this.targetUrl = s;
          this.isTargetUrlOk = true;
          continue;
        }
      }

      // Validity check
      if ((this.maxPages <= 0) || (this.maxPages > MAX_PAGE_LIMIT)) {
        this.isMaxPagesOk = false;
      }
      this.isOk = (this.isOk && this.isTargetUrlOk && this.isMaxPagesOk);
    }

    
    /**
     * Provides a string dump of all set fields from parsing.
     * @return String dump of all fields and flags. 
     */
    @Override
    public String toString() {

      String colonStr = ": ";
      String equalStr = " = ";
      String commaStr = ", ";

      String s = WebSpider.RESULT_TAG_LIST[0] + colonStr + WebSpider.RESULT_TAG_LIST[1] + equalStr
          + this.isHelpOn.toString() + commaStr + WebSpider.RESULT_TAG_LIST[2] + equalStr
          + this.isTestParseOn.toString() + commaStr + WebSpider.RESULT_TAG_LIST[3] + equalStr
          + this.isTotalLinksOn.toString() + commaStr + WebSpider.RESULT_TAG_LIST[4] + equalStr
          + this.isMostPopularOn.toString() + commaStr + WebSpider.RESULT_TAG_LIST[5] + equalStr
          + this.isLoggingOn.toString() + commaStr + WebSpider.RESULT_TAG_LIST[6] + equalStr
          + this.isMaxPagesOk.toString() + commaStr + WebSpider.RESULT_TAG_LIST[7] + equalStr
          + this.isTargetUrlOk.toString() + commaStr + WebSpider.RESULT_TAG_LIST[8] + equalStr
          + this.targetUrl + commaStr + WebSpider.RESULT_TAG_LIST[9] + equalStr 
          + this.maxPages + commaStr + WebSpider.RESULT_TAG_LIST[10] + equalStr 
          + this.isOk;
      return s;
    }
  }

  /**
   * Stores URL and the links pointing to it for use in determining the most popular link.
   * 
   * @author rcox
   */
  public static class UrlList {
    List<UrlItem> urlItems;

    /**
     * Initializes list.
     */
    public UrlList() {
      this.urlItems = new ArrayList<UrlItem>();
    }

    /**
     * Either adds URL to list, or increments count if URL is already present.
     * 
     * @param newUrl Url to add to list.
     */
    public void add(String newUrl) {
      Boolean found = false;
      for (UrlItem item : this.urlItems) {
        if (item.url.equalsIgnoreCase(newUrl)) {
          found = true;
          item.count++;
        }
      }
      if (!found) {
        UrlItem newItem = new UrlItem(newUrl);
        this.urlItems.add(newItem);
      }
    }

    /**
     * Search through internal data and find the most popular URL.
     * 
     * @return UrlItem of most popular URL
     */
    public UrlItem getMostPopular() {
      UrlItem maxItem = new UrlItem("");
      for (UrlItem item : this.urlItems) {
        if (item.count > maxItem.count) {
          maxItem.count = item.count;
          maxItem.url = item.url;
        }
      }
      return maxItem;
    }
  }

  /**
   * Stores a single URL with associated number of links pointing to it.
   * 
   * @author rcox
   */
  public static class UrlItem {
    
    /**
     * URL to store.
     */
    public String url;
    
    /**
     * Count of links pointing to URL.
     */
    public Integer count;

    /**
     * Creates new item with URL and count 0.
     * @param url URL string to populate item.
     */
    public UrlItem(String url) {
      this.url = url;
      this.count = 1;
    }

    /**
     * Equals function when URL strings are the same.
     * @param urlItem the item to compare with.
     * @return True if URL strings are equal.
     */
    @Override
    public boolean equals(Object urlItem) {
      if (urlItem == null) {
        return false;
      }
      if (!(urlItem instanceof UrlItem)) {
        return false;
      }
      return this.url.equalsIgnoreCase(((UrlItem) urlItem).url);
    }

    /**
     * Hash code for item.
     * @return hash code of item
     */
    @Override
    public int hashCode() {
      return this.url.hashCode();
    }
  }

  /**
   * Stores statistics collected by the application with searching through URLs. 
   * Currently the information stored is: number of links found, the most popular URL, 
   * and the number of links pointing to the most popular URL.
   * 
   * @author rcox
   */
  public static class UrlStats {
    
    /**
     * Total links.
     */
    public Integer linkCount;
    
    /**
     * URL of most popular link.
     */
    public String mostPopularUrl;
    
    /**
     * Number of links pointing to most popular URL.
     */
    public Integer mostPopularLinkCount;

    /**
     * Creates an empty stat item.
     */
    public UrlStats() {
      this.linkCount = 0;
      this.mostPopularUrl = "";
      this.mostPopularLinkCount = 0;
    }
  }

  /**
   * Custom formatter used by the logger to format the output string appropriately.
   * 
   * @author rcox
   */
  public static class WebSpiderFormatter extends Formatter {

    /**
     * Formats log record to HH:MM:SS Message.
     * @param record Log record to extract information to place in formatted string.
     * @return resulting string in the HH:MM:SS Message format.
     */
    public String format(LogRecord record) {
      Long msec = record.getMillis();
      Date date = new Date();
      date.setTime(msec);
      String message = date.toString();
      message = message.substring(10, 19);
      message = message.concat(" " + record.getMessage() + "\r\n");
      return message;
    }
  }
  
  /** 
   * Handler to send log output to standard output.
   */
   static class StandardHandler extends Handler {
    
    /**
     * Since this handler just prints to standard output, nothing to close.
     */
    public void close() {
      // Empty
    }
    
    /**
     * Nothing the flush.
     */
    public void flush() {
      // Empty
    }
    
    /** 
     * Sends output to standard out.
     * @param record LogRecord to format and print. 
     */
    public void publish(LogRecord record) {
      this.flush();
      WebSpiderFormatter formatter = new WebSpiderFormatter();
      String outputString = formatter.formatMessage(record);
      System.out.println(outputString);
    }
  }

}
